{"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"widgets":{"application/vnd.jupyter.widget-state+json":{"0fff4a6672c848a38945e19101c46168":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1d17b8847ab34a1d8fc79430e8b64a12":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_e5c8b5933c754a1192456f43682276c5","placeholder":"​","style":"IPY_MODEL_84abf77081a04bb686f794b49e04761d","value":" 1/1 [00:00&lt;00:00, 35.79ba/s]"}},"207785da1f404d6ea0e8c63655a7aa51":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"22fc25617c664e4aa02b7457832c1bef":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"256a0dbb2f104d928e181d2a882a9867":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"2add9d30edd84152bb6d7bfa4ed2d910":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"31390facfbeb455fa83067fc23d30718":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5367cbe7ca4d43b089ff0d5d7cd17d3c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b53ebcf672a244978391cae559db5bf2","placeholder":"​","style":"IPY_MODEL_f2de6a2c1f4b4a2fa5196028c0da0754","value":" 1/1 [00:00&lt;00:00, 18.79ba/s]"}},"56098a347ea94ea3b10bd8d2ec0d4288":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"567176b50d074a1c9d384a8df8c3ff4c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_f34ffd11b98045b9bae326dd48a54896","placeholder":"​","style":"IPY_MODEL_e3e19fc9963c4bf39293bda7c2030d5a","value":"Creating CSV from Arrow format: 100%"}},"632135599e39470aac1a3bb3d3de0ca4":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_567176b50d074a1c9d384a8df8c3ff4c","IPY_MODEL_6bfa39f2c84e4f08b5ffb9a416398824","IPY_MODEL_911b40169865413b98643789150e5495"],"layout":"IPY_MODEL_2add9d30edd84152bb6d7bfa4ed2d910"}},"65f75f3d72cd415faffb32999893738d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_0fff4a6672c848a38945e19101c46168","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_de662cb67d054f08a15d191923d40369","value":1}},"68790cfe4ea14fc4a63275f3e99c468f":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6bfa39f2c84e4f08b5ffb9a416398824":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_c15c27b0523a429e9d77f439d47ada90","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_256a0dbb2f104d928e181d2a882a9867","value":1}},"6e7fb9a1d1454fcd9bcf5b5f748fb975":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"71237d176b2c4138a5e0346d10482257":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"761e3c6035bf49429b3035145451d2df":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_207785da1f404d6ea0e8c63655a7aa51","placeholder":"​","style":"IPY_MODEL_71237d176b2c4138a5e0346d10482257","value":"Creating CSV from Arrow format: 100%"}},"7940d9e3f5fa4592b58dec3fdb55595a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"84abf77081a04bb686f794b49e04761d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"86761a36bdb04fed9f1dae6e74da54ce":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_e079c0eb3d9c4ae3a40491f059a864f7","IPY_MODEL_d4bde97f3895450782030cad7808ea59","IPY_MODEL_1d17b8847ab34a1d8fc79430e8b64a12"],"layout":"IPY_MODEL_fb953d8d116e4cc389dbc23a0055873f"}},"911b40169865413b98643789150e5495":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_68790cfe4ea14fc4a63275f3e99c468f","placeholder":"​","style":"IPY_MODEL_e7df092ac205443e8baa3646ff1eae5b","value":" 1/1 [00:00&lt;00:00, 33.41ba/s]"}},"95d92c5c74e845779337eb727c2bbfc0":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_56098a347ea94ea3b10bd8d2ec0d4288","placeholder":"​","style":"IPY_MODEL_7940d9e3f5fa4592b58dec3fdb55595a","value":" 1/1 [00:00&lt;00:00, 33.86ba/s]"}},"b4a71cf676584200b46922fb976d1d50":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_22fc25617c664e4aa02b7457832c1bef","placeholder":"​","style":"IPY_MODEL_b556054df18d47aa8cb58ac482ca31bb","value":"Creating CSV from Arrow format: 100%"}},"b53ebcf672a244978391cae559db5bf2":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b556054df18d47aa8cb58ac482ca31bb":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"bf27f78edb15477492fb2d5dd8dc5137":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"bf7f045bdbe24360ad2aa2f4c8f02e79":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_761e3c6035bf49429b3035145451d2df","IPY_MODEL_ffb7c97e7af648aaa13a43427154140e","IPY_MODEL_95d92c5c74e845779337eb727c2bbfc0"],"layout":"IPY_MODEL_6e7fb9a1d1454fcd9bcf5b5f748fb975"}},"c077837b8b044c2fb14dcd8fbc44a37b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"c15c27b0523a429e9d77f439d47ada90":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c4bc64d47f2e4a239cf7156e7812887d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_b4a71cf676584200b46922fb976d1d50","IPY_MODEL_65f75f3d72cd415faffb32999893738d","IPY_MODEL_5367cbe7ca4d43b089ff0d5d7cd17d3c"],"layout":"IPY_MODEL_31390facfbeb455fa83067fc23d30718"}},"c9f479f81119450bb451d5830361467c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"d4bde97f3895450782030cad7808ea59":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_e556e254882340168833ecf1a7153a90","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c077837b8b044c2fb14dcd8fbc44a37b","value":1}},"de662cb67d054f08a15d191923d40369":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"e079c0eb3d9c4ae3a40491f059a864f7":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_bf27f78edb15477492fb2d5dd8dc5137","placeholder":"​","style":"IPY_MODEL_ee96a569b8a54944bcac1b1bc164e53e","value":"Creating CSV from Arrow format: 100%"}},"e3e19fc9963c4bf39293bda7c2030d5a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"e556e254882340168833ecf1a7153a90":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e5c8b5933c754a1192456f43682276c5":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e7df092ac205443e8baa3646ff1eae5b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ed457547dc154f6bbce4ec970bb09c76":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ee96a569b8a54944bcac1b1bc164e53e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"f2de6a2c1f4b4a2fa5196028c0da0754":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"f34ffd11b98045b9bae326dd48a54896":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"fb953d8d116e4cc389dbc23a0055873f":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ffb7c97e7af648aaa13a43427154140e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_ed457547dc154f6bbce4ec970bb09c76","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c9f479f81119450bb451d5830361467c","value":1}}}},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 使用LangChain和RAGAS对RAG系统进行自动有效评估\n\n我们主要讨论一下LLM RAG问答系统中一个重要的组成部分:\n\n- Evaluation\n\n我们主要使用LangChain 构建RAG问答系统，利用 RAGAS 框架进行评估，因为它正逐渐成为评估 RAG 系统的标准方法","metadata":{"id":"wa8ykQk92aLX"}},{"cell_type":"markdown","source":"### 首先安装 依赖","metadata":{}},{"cell_type":"code","source":"!pip install -U -q langchain openai ragas arxiv pymupdf chromadb wandb tiktoken","metadata":{"id":"5BN13TZlSCv4","execution":{"iopub.status.busy":"2024-05-10T19:52:34.913776Z","iopub.execute_input":"2024-05-10T19:52:34.914382Z","iopub.status.idle":"2024-05-10T19:53:31.961999Z","shell.execute_reply.started":"2024-05-10T19:52:34.914347Z","shell.execute_reply":"2024-05-10T19:53:31.961028Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\ncudf 23.8.0 requires cubinlinker, which is not installed.\ncudf 23.8.0 requires cupy-cuda11x>=12.0.0, which is not installed.\ncudf 23.8.0 requires ptxcompiler, which is not installed.\ncuml 23.8.0 requires cupy-cuda11x>=12.0.0, which is not installed.\ndask-cudf 23.8.0 requires cupy-cuda11x>=12.0.0, which is not installed.\nkeras-cv 0.8.2 requires keras-core, which is not installed.\nkeras-nlp 0.9.3 requires keras-core, which is not installed.\ntensorflow-decision-forests 1.8.1 requires wurlitzer, which is not installed.\napache-beam 2.46.0 requires dill<0.3.2,>=0.3.1.1, but you have dill 0.3.8 which is incompatible.\napache-beam 2.46.0 requires numpy<1.25.0,>=1.14.3, but you have numpy 1.26.4 which is incompatible.\napache-beam 2.46.0 requires pyarrow<10.0.0,>=3.0.0, but you have pyarrow 15.0.2 which is incompatible.\ncudf 23.8.0 requires cuda-python<12.0a0,>=11.7.1, but you have cuda-python 12.4.0 which is incompatible.\ncudf 23.8.0 requires pandas<1.6.0dev0,>=1.3, but you have pandas 2.1.4 which is incompatible.\ncudf 23.8.0 requires protobuf<5,>=4.21, but you have protobuf 3.20.3 which is incompatible.\ncudf 23.8.0 requires pyarrow==11.*, but you have pyarrow 15.0.2 which is incompatible.\ncuml 23.8.0 requires dask==2023.7.1, but you have dask 2024.4.1 which is incompatible.\ndask-cuda 23.8.0 requires dask==2023.7.1, but you have dask 2024.4.1 which is incompatible.\ndask-cuda 23.8.0 requires pandas<1.6.0dev0,>=1.3, but you have pandas 2.1.4 which is incompatible.\ndask-cudf 23.8.0 requires dask==2023.7.1, but you have dask 2024.4.1 which is incompatible.\ndask-cudf 23.8.0 requires pandas<1.6.0dev0,>=1.3, but you have pandas 2.1.4 which is incompatible.\ndistributed 2023.7.1 requires dask==2023.7.1, but you have dask 2024.4.1 which is incompatible.\ngoogle-cloud-bigquery 2.34.4 requires packaging<22.0dev,>=14.3, but you have packaging 23.2 which is incompatible.\njupyterlab 4.1.6 requires jupyter-lsp>=2.0.0, but you have jupyter-lsp 1.5.1 which is incompatible.\njupyterlab-lsp 5.1.0 requires jupyter-lsp>=2.0.0, but you have jupyter-lsp 1.5.1 which is incompatible.\nkfp 2.5.0 requires google-cloud-storage<3,>=2.2.1, but you have google-cloud-storage 1.44.0 which is incompatible.\nkfp 2.5.0 requires kubernetes<27,>=8.0.0, but you have kubernetes 29.0.0 which is incompatible.\nlibpysal 4.9.2 requires shapely>=2.0.1, but you have shapely 1.8.5.post1 which is incompatible.\nmomepy 0.7.0 requires shapely>=2, but you have shapely 1.8.5.post1 which is incompatible.\nosmnx 1.9.2 requires shapely>=2.0, but you have shapely 1.8.5.post1 which is incompatible.\nraft-dask 23.8.0 requires dask==2023.7.1, but you have dask 2024.4.1 which is incompatible.\nspopt 0.6.0 requires shapely>=2.0.1, but you have shapely 1.8.5.post1 which is incompatible.\ntensorflow 2.15.0 requires keras<2.16,>=2.15.0, but you have keras 3.2.1 which is incompatible.\nydata-profiling 4.6.4 requires numpy<1.26,>=1.16.0, but you have numpy 1.26.4 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"import os\nimport openai\nfrom getpass import getpass\n\nopenai.api_key = getpass(\"Please provide your OpenAI Key: \")\nos.environ[\"OPENAI_API_KEY\"] = openai.api_key","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"8Lhqp5rUThG-","outputId":"c33f7eee-b819-40bd-dc75-ce90721a6a94","execution":{"iopub.status.busy":"2024-05-10T19:53:31.963770Z","iopub.execute_input":"2024-05-10T19:53:31.964073Z","iopub.status.idle":"2024-05-10T19:53:38.294347Z","shell.execute_reply.started":"2024-05-10T19:53:31.964045Z","shell.execute_reply":"2024-05-10T19:53:38.293304Z"},"trusted":true},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdin","text":"Please provide your OpenAI Key:  ························································\n"}]},{"cell_type":"markdown","source":"### 数据准备\n\n主要以Arxiv的论文为例进行评估，通过 `ArxivLoader` 加载数据(论文)作为RAG的上下文。\n\n- [`ArxivLoader`](https://api.python.langchain.com/en/latest/document_loaders/langchain_community.document_loaders.arxiv.ArxivLoader.html#langchain_community.document_loaders.arxiv.ArxivLoader)\n","metadata":{"id":"DV_BOewX8CW0"}},{"cell_type":"code","source":"from langchain.document_loaders import ArxivLoader\n\npaper_docs = ArxivLoader(query=\"2309.15217\", load_max_docs=1).load()\nlen(paper_docs)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"DTDNFXaBSO2j","outputId":"3b24521d-5c6f-466b-d818-46ce68d359ee","execution":{"iopub.status.busy":"2024-05-10T19:53:46.440225Z","iopub.execute_input":"2024-05-10T19:53:46.440739Z","iopub.status.idle":"2024-05-10T19:53:47.468053Z","shell.execute_reply.started":"2024-05-10T19:53:46.440708Z","shell.execute_reply":"2024-05-10T19:53:47.467081Z"},"trusted":true},"execution_count":3,"outputs":[{"execution_count":3,"output_type":"execute_result","data":{"text/plain":"1"},"metadata":{}}]},{"cell_type":"code","source":"for doc in paper_docs:\n  print(doc.metadata)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"nNPAWPgNSyGP","outputId":"b2f80fc8-792c-489a-b8d4-9f98678c679a","execution":{"iopub.status.busy":"2024-05-10T19:53:49.357162Z","iopub.execute_input":"2024-05-10T19:53:49.357587Z","iopub.status.idle":"2024-05-10T19:53:49.363098Z","shell.execute_reply.started":"2024-05-10T19:53:49.357555Z","shell.execute_reply":"2024-05-10T19:53:49.362067Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stdout","text":"{'Published': '2023-09-26', 'Title': 'RAGAS: Automated Evaluation of Retrieval Augmented Generation', 'Authors': 'Shahul Es, Jithin James, Luis Espinosa-Anke, Steven Schockaert', 'Summary': 'We introduce RAGAs (Retrieval Augmented Generation Assessment), a framework\\nfor reference-free evaluation of Retrieval Augmented Generation (RAG)\\npipelines. RAG systems are composed of a retrieval and an LLM based generation\\nmodule, and provide LLMs with knowledge from a reference textual database,\\nwhich enables them to act as a natural language layer between a user and\\ntextual databases, reducing the risk of hallucinations. Evaluating RAG\\narchitectures is, however, challenging because there are several dimensions to\\nconsider: the ability of the retrieval system to identify relevant and focused\\ncontext passages, the ability of the LLM to exploit such passages in a faithful\\nway, or the quality of the generation itself. With RAGAs, we put forward a\\nsuite of metrics which can be used to evaluate these different dimensions\\n\\\\textit{without having to rely on ground truth human annotations}. We posit\\nthat such a framework can crucially contribute to faster evaluation cycles of\\nRAG architectures, which is especially important given the fast adoption of\\nLLMs.'}\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### 创建RAG文本分割、Embedding model 、 向量库存储\n\n我们主要使用 `RecursiveCharacterTextSplitter` 切割文本，通过`OpenAIEmbeddings()`进行文本编码，存储到 `VectorStore`。\n\n- `RecursiveCharacterTextSplitter()`\n- `OpenAIEmbeddings()`\n- `Chroma`","metadata":{"id":"Z7ht6bJX9PAY"}},{"cell_type":"code","source":"from langchain.vectorstores import Chroma\nfrom langchain.embeddings import OpenAIEmbeddings\nfrom langchain.text_splitter import RecursiveCharacterTextSplitter\n\ntext_splitter = RecursiveCharacterTextSplitter(chunk_size=500)\n\ndocs = text_splitter.split_documents(paper_docs)\n\nvectorstore = Chroma.from_documents(docs, OpenAIEmbeddings())","metadata":{"id":"xne8P5dQTUiR","execution":{"iopub.status.busy":"2024-05-10T19:54:04.685305Z","iopub.execute_input":"2024-05-10T19:54:04.686212Z","iopub.status.idle":"2024-05-10T19:54:08.946111Z","shell.execute_reply.started":"2024-05-10T19:54:04.686174Z","shell.execute_reply":"2024-05-10T19:54:08.945045Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:119: LangChainDeprecationWarning: The class `OpenAIEmbeddings` was deprecated in LangChain 0.0.9 and will be removed in 0.3.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import OpenAIEmbeddings`.\n  warn_deprecated(\n","output_type":"stream"}]},{"cell_type":"code","source":"len(docs)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cnRzYx4c_2mZ","outputId":"59d9bdd8-0414-4e8b-c285-bf3a2760e26a","execution":{"iopub.status.busy":"2024-05-10T19:54:12.106827Z","iopub.execute_input":"2024-05-10T19:54:12.107197Z","iopub.status.idle":"2024-05-10T19:54:12.113731Z","shell.execute_reply.started":"2024-05-10T19:54:12.107166Z","shell.execute_reply":"2024-05-10T19:54:12.112753Z"},"trusted":true},"execution_count":6,"outputs":[{"execution_count":6,"output_type":"execute_result","data":{"text/plain":"107"},"metadata":{}}]},{"cell_type":"code","source":"print(max([len(chunk.page_content) for chunk in docs]))","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"WyUh8EVI_6TZ","outputId":"643fca9d-77c0-4296-d953-ec62d6de8954","execution":{"iopub.status.busy":"2024-05-10T19:54:13.901334Z","iopub.execute_input":"2024-05-10T19:54:13.901706Z","iopub.status.idle":"2024-05-10T19:54:13.907099Z","shell.execute_reply.started":"2024-05-10T19:54:13.901676Z","shell.execute_reply":"2024-05-10T19:54:13.906132Z"},"trusted":true},"execution_count":7,"outputs":[{"name":"stdout","text":"497\n","output_type":"stream"}]},{"cell_type":"markdown","source":"现在我们可以利用 `Chroma` 向量库的 `.as_retriever()` 方式进行检索，需要控制的主要参数为 `k`","metadata":{"id":"0f9kNIUUTxdT"}},{"cell_type":"code","source":"base_retriever = vectorstore.as_retriever(search_kwargs={\"k\" : 3})","metadata":{"id":"bwbdftltT29h","execution":{"iopub.status.busy":"2024-05-10T19:54:21.241106Z","iopub.execute_input":"2024-05-10T19:54:21.242202Z","iopub.status.idle":"2024-05-10T19:54:21.246987Z","shell.execute_reply.started":"2024-05-10T19:54:21.242157Z","shell.execute_reply":"2024-05-10T19:54:21.245951Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"relevant_docs = base_retriever.get_relevant_documents(\"What is Retrieval Augmented Generation?\")","metadata":{"id":"r0Pie4xqUCkW","execution":{"iopub.status.busy":"2024-05-10T19:54:23.146375Z","iopub.execute_input":"2024-05-10T19:54:23.147124Z","iopub.status.idle":"2024-05-10T19:54:23.377555Z","shell.execute_reply.started":"2024-05-10T19:54:23.147091Z","shell.execute_reply":"2024-05-10T19:54:23.376602Z"},"trusted":true},"execution_count":10,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:119: LangChainDeprecationWarning: The method `BaseRetriever.get_relevant_documents` was deprecated in langchain-core 0.1.46 and will be removed in 0.3.0. Use invoke instead.\n  warn_deprecated(\n","output_type":"stream"}]},{"cell_type":"code","source":"len(relevant_docs)","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Z_CiHTD0UKj7","outputId":"fab040cf-971f-440a-a6aa-93873c8e152f","execution":{"iopub.status.busy":"2024-05-10T19:54:25.853921Z","iopub.execute_input":"2024-05-10T19:54:25.854415Z","iopub.status.idle":"2024-05-10T19:54:25.860784Z","shell.execute_reply.started":"2024-05-10T19:54:25.854372Z","shell.execute_reply":"2024-05-10T19:54:25.859862Z"},"trusted":true},"execution_count":11,"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":"3"},"metadata":{}}]},{"cell_type":"markdown","source":"### 创建prompt ——— 生成答案\n我们需要利用`LLM`对`Context` 生成一系列的问题的`answer`\n","metadata":{}},{"cell_type":"code","source":"from langchain import PromptTemplate\n\ntemplate = \"\"\"You are an assistant for question-answering tasks. \nUse the following pieces of retrieved context to answer the question. \nIf you don't know the answer, just say that you don't know. \n\nQuestion: {question} \n\nContext: {context} \n\nAnswer:\n\"\"\"\n\nprompt = PromptTemplate(\n    template=template, \n    input_variables=[\"context\",\"question\"]\n  )\n\nprint(prompt)","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:55:14.518601Z","iopub.execute_input":"2024-05-10T19:55:14.518988Z","iopub.status.idle":"2024-05-10T19:55:14.650176Z","shell.execute_reply.started":"2024-05-10T19:55:14.518958Z","shell.execute_reply":"2024-05-10T19:55:14.649153Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stdout","text":"input_variables=['context', 'question'] template=\"You are an assistant for question-answering tasks. \\nUse the following pieces of retrieved context to answer the question. \\nIf you don't know the answer, just say that you don't know. \\n\\nQuestion: {question} \\n\\nContext: {context} \\n\\nAnswer:\\n\"\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### 生成`answer`,利用LLM\n利用 `Runnable` 定义一个 `chain` 实现rag全流程。","metadata":{}},{"cell_type":"code","source":"from langchain.schema.runnable import RunnablePassthrough\nfrom langchain.schema.output_parser import StrOutputParser\nfrom langchain.chat_models import ChatOpenAI\n\nllm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n\nrag_chain = (\n    {\"context\": base_retriever,  \"question\": RunnablePassthrough()} \n    | prompt \n    | llm\n    | StrOutputParser() \n)","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:55:34.967750Z","iopub.execute_input":"2024-05-10T19:55:34.968193Z","iopub.status.idle":"2024-05-10T19:55:35.026050Z","shell.execute_reply.started":"2024-05-10T19:55:34.968163Z","shell.execute_reply":"2024-05-10T19:55:35.025099Z"},"trusted":true},"execution_count":13,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:119: LangChainDeprecationWarning: The class `ChatOpenAI` was deprecated in LangChain 0.0.10 and will be removed in 0.3.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import ChatOpenAI`.\n  warn_deprecated(\n","output_type":"stream"}]},{"cell_type":"markdown","source":"#### 创建 RAGAs 所需的数据\nquestion  Answer   contexts  ground_truths","metadata":{}},{"cell_type":"code","source":"# Ragas 数据集格式要求  ['question', 'answer', 'contexts', 'ground_truths']\n'''\n{\n    \"question\": [], <-- 问题基于Context的\n    \"answer\": [], <-- 答案基于LLM生成的\n    \"contexts\": [], <-- context\n    \"ground_truths\": [] <-- 标准答案\n}\n'''\n\nfrom datasets import Dataset\n\nquestions = [\"What is faithfulness ?\", \n             \"How many pages are included in the WikiEval dataset, and which years do they cover information from?\",\n             \"Why is evaluating Retrieval Augmented Generation (RAG) systems challenging?\",\n            ]\nground_truths = [[\"Faithfulness refers to the idea that the answer should be grounded in the given context.\"],\n                 [\" To construct the dataset, we first selected 50 Wikipedia pages covering events that have happened since the start of 2022.\"],\n                [\"Evaluating RAG architectures is, however, challenging because there are several dimensions to consider: the ability of the retrieval system to identify relevant and focused context passages, the ability of the LLM to exploit such passages in a faithful way, or the quality of the generation itself.\"]]\nanswers = []\ncontexts = []\n\n# 生成答案\nfor query in questions:\n    answers.append(rag_chain.invoke(query))\n    contexts.append([docs.page_content for docs in base_retriever.get_relevant_documents(query)])\n\n# 构建数据\ndata = {\n    \"question\": questions,\n    \"answer\": answers,\n    \"contexts\": contexts,\n    \"ground_truths\": ground_truths\n}\ndataset = Dataset.from_dict(data)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:55:46.448366Z","iopub.execute_input":"2024-05-10T19:55:46.449475Z","iopub.status.idle":"2024-05-10T19:55:53.416244Z","shell.execute_reply.started":"2024-05-10T19:55:46.449436Z","shell.execute_reply":"2024-05-10T19:55:53.415397Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"dataset","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:55:53.425364Z","iopub.execute_input":"2024-05-10T19:55:53.425967Z","iopub.status.idle":"2024-05-10T19:55:53.433450Z","shell.execute_reply.started":"2024-05-10T19:55:53.425939Z","shell.execute_reply":"2024-05-10T19:55:53.432570Z"},"trusted":true},"execution_count":16,"outputs":[{"execution_count":16,"output_type":"execute_result","data":{"text/plain":"Dataset({\n    features: ['question', 'answer', 'contexts', 'ground_truths'],\n    num_rows: 3\n})"},"metadata":{}}]},{"cell_type":"markdown","source":"### 使用RAGAs 进行评估","metadata":{}},{"cell_type":"code","source":"from ragas import evaluate\nfrom ragas.metrics import (\n    faithfulness,\n    answer_relevancy,\n    context_relevancy,\n    context_recall,\n    context_precision,\n)\n\nresult = evaluate(\n    dataset = dataset, \n    metrics=[\n        context_precision,\n        context_recall,\n        faithfulness,\n        answer_relevancy,\n    ],\n)\n\nresult","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:56:16.623057Z","iopub.execute_input":"2024-05-10T19:56:16.623771Z","iopub.status.idle":"2024-05-10T19:56:26.220039Z","shell.execute_reply.started":"2024-05-10T19:56:16.623738Z","shell.execute_reply":"2024-05-10T19:56:26.219039Z"},"trusted":true},"execution_count":17,"outputs":[{"output_type":"display_data","data":{"text/plain":"Evaluating:   0%|          | 0/12 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"273450aeb39440fe9f06192f893dae06"}},"metadata":{}},{"execution_count":17,"output_type":"execute_result","data":{"text/plain":"{'context_precision': 0.9444, 'context_recall': 1.0000, 'faithfulness': 1.0000, 'answer_relevancy': 0.8745}"},"metadata":{}}]},{"cell_type":"code","source":"import pandas as pd\npd.set_option(\"display.max_colwidth\", None)\n\ndf = result.to_pandas()\ndf","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:56:26.294177Z","iopub.execute_input":"2024-05-10T19:56:26.294548Z","iopub.status.idle":"2024-05-10T19:56:26.320062Z","shell.execute_reply.started":"2024-05-10T19:56:26.294522Z","shell.execute_reply":"2024-05-10T19:56:26.319031Z"},"trusted":true},"execution_count":20,"outputs":[{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"                                                                                               question  \\\n0                                                                                What is faithfulness ?   \n1  How many pages are included in the WikiEval dataset, and which years do they cover information from?   \n2                           Why is evaluating Retrieval Augmented Generation (RAG) systems challenging?   \n\n                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               answer  \\\n0                                                                                                                                                                                                                                                                                                                                                                                                                                           Faithfulness refers to the idea that the answer should be grounded in the given context to avoid hallucinations and ensure factual consistency with the sources provided.   \n1                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        The WikiEval dataset includes 50 Wikipedia pages covering events that have happened since the start of 2022.   \n2  Evaluating Retrieval Augmented Generation (RAG) systems is challenging because there are several dimensions to consider. These include the ability of the retrieval system to identify relevant and focused context passages, the ability of the Language Model (LLM) to exploit such passages faithfully, and the quality of the generation itself. Additionally, the overall performance of RAG systems can be affected by various factors such as the retrieval model, the considered corpus, the LM, or the prompt formulation, among others. This complexity makes evaluating RAG systems a challenging task.   \n\n                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            contexts  \\\n0                                                                                         [First, Faithfulness refers to the idea that the an-\\nswer should be grounded in the given context. This\\nis important to avoid hallucinations, and to ensure\\nthat the retrieved context can act as a justification\\nfor the generated answer. Indeed, RAG systems are\\noften used in applications where the factual con-\\nsistency of the generated text w.r.t. the grounded\\nsources is highly important, e.g. in domains such as\\nlaw, where information is constantly evolving. Sec-, Faithfulness\\nTo obtain human judgements about\\nfaithfulness, we first used ChatGPT to answer the\\nquestion without access to any additional context.\\nWe then asked the annotators to judge which of the\\ntwo answers was the most faithful (i.e. the standard\\none or the one generated without context), given\\nthe question and corresponding Wikipedia page.\\nAnswer relevance\\nWe first used ChatGPT to\\nobtain candidate answers with lower answer rel-\\nevance, using the following prompt:, an answer as(q). When building a RAG system,\\nwe usually do not have access to human-annotated\\ndatasets or reference answers. We therefore fo-\\ncus on metrics that are fully self-contained and\\nreference-free. We focus in particular three quality\\naspects, which we argue are of central importance.\\nFirst, Faithfulness refers to the idea that the an-\\nswer should be grounded in the given context. This\\nis important to avoid hallucinations, and to ensure]   \n1                                [which we refer to as WikiEval4. To construct the\\ndataset, we first selected 50 Wikipedia pages cov-\\nering events that have happened since the start of\\n20225. In selecting these pages, we prioritised\\nthose with recent edits. For each of the 50 pages,\\nwe then asked ChatGPT to suggest a question that\\ncan be answered based on the introductory section\\nof the page, using the following prompt:\\nYour task is to formulate a question from\\ngiven context satisfying the rules given\\nbelow:, which are annotated with human judgments. We\\ncan then verify to what extent our metrics agree\\nwith human assessments of faithfulness, answer\\nrelevance and context relevance. Since we are not\\naware of any publicly available datasets that could\\nbe used for this purpose, we created a new dataset,\\nwhich we refer to as WikiEval4. To construct the\\ndataset, we first selected 50 Wikipedia pages cov-\\nering events that have happened since the start of\\n20225. In selecting these pages, we prioritised, cessing (EMNLP-IJCNLP), pages 563–578, Hong\\nKong, China. Association for Computational Lin-\\nguistics.\\nA\\nExamples from WikiEval\\nTables 2, 3 and 4 show examples from the WikiEval\\ndataset, focusing in particular on answers with high\\nand low faithfulness (Table 2), high and low answer\\nrelevance (Table 3), and high and low context rele-\\nvance (Table 4).\\nQuestion\\nContext\\nAnswer\\nWho directed the film Op-\\npenheimer and who stars\\nas J. Robert Oppenheimer\\nin the film?]   \n2  [retrieval-augmented systems is thus paramount. In\\npractice, RAG systems are often evaluated in terms\\nof the language modelling task itself, i.e. by mea-\\nsuring perplexity on some reference corpus. How-\\never, such evaluations are not always predictive\\nof downstream performance (Wang et al., 2023c).\\nMoreover, this evaluation strategy relies on the LM\\nprobabilities, which are not accessible for some\\nclosed models (e.g. ChatGPT and GPT-4). Ques-\\ntion answering is another common evaluation task,, that are only available through APIs.\\nWhile the usefulness of retrieval-augmented\\nstrategies is clear, their implementation requires\\na significant amount of tuning, as the overall per-\\nformance will be affected by the retrieval model,\\nthe considered corpus, the LM, or the prompt for-\\nmulation, among others. Automated evaluation of\\nretrieval-augmented systems is thus paramount. In\\npractice, RAG systems are often evaluated in terms\\nof the language modelling task itself, i.e. by mea-, Abstract\\nWe introduce RAGAS (Retrieval Augmented\\nGeneration Assessment), a framework for\\nreference-free evaluation of Retrieval Aug-\\nmented Generation (RAG) pipelines.\\nRAG\\nsystems are composed of a retrieval and an\\nLLM based generation module, and provide\\nLLMs with knowledge from a reference textual\\ndatabase, which enables them to act as a natu-\\nral language layer between a user and textual\\ndatabases, reducing the risk of hallucinations.\\nEvaluating RAG architectures is, however, chal-]   \n\n                                                                                                                                                                                                                                                                                                  ground_truths  \\\n0                                                                                                                                                                                                                    [Faithfulness refers to the idea that the answer should be grounded in the given context.]   \n1                                                                                                                                                                                 [ To construct the dataset, we first selected 50 Wikipedia pages covering events that have happened since the start of 2022.]   \n2  [Evaluating RAG architectures is, however, challenging because there are several dimensions to consider: the ability of the retrieval system to identify relevant and focused context passages, the ability of the LLM to exploit such passages in a faithful way, or the quality of the generation itself.]   \n\n                                                                                                                                                                                                                                                                                                 ground_truth  \\\n0                                                                                                                                                                                                                    Faithfulness refers to the idea that the answer should be grounded in the given context.   \n1                                                                                                                                                                                  To construct the dataset, we first selected 50 Wikipedia pages covering events that have happened since the start of 2022.   \n2  Evaluating RAG architectures is, however, challenging because there are several dimensions to consider: the ability of the retrieval system to identify relevant and focused context passages, the ability of the LLM to exploit such passages in a faithful way, or the quality of the generation itself.   \n\n   context_precision  context_recall  faithfulness  answer_relevancy  \n0           0.833333             1.0           1.0          0.953604  \n1           1.000000             1.0           1.0          0.728021  \n2           1.000000             1.0           1.0          0.941939  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>question</th>\n      <th>answer</th>\n      <th>contexts</th>\n      <th>ground_truths</th>\n      <th>ground_truth</th>\n      <th>context_precision</th>\n      <th>context_recall</th>\n      <th>faithfulness</th>\n      <th>answer_relevancy</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>What is faithfulness ?</td>\n      <td>Faithfulness refers to the idea that the answer should be grounded in the given context to avoid hallucinations and ensure factual consistency with the sources provided.</td>\n      <td>[First, Faithfulness refers to the idea that the an-\\nswer should be grounded in the given context. This\\nis important to avoid hallucinations, and to ensure\\nthat the retrieved context can act as a justification\\nfor the generated answer. Indeed, RAG systems are\\noften used in applications where the factual con-\\nsistency of the generated text w.r.t. the grounded\\nsources is highly important, e.g. in domains such as\\nlaw, where information is constantly evolving. Sec-, Faithfulness\\nTo obtain human judgements about\\nfaithfulness, we first used ChatGPT to answer the\\nquestion without access to any additional context.\\nWe then asked the annotators to judge which of the\\ntwo answers was the most faithful (i.e. the standard\\none or the one generated without context), given\\nthe question and corresponding Wikipedia page.\\nAnswer relevance\\nWe first used ChatGPT to\\nobtain candidate answers with lower answer rel-\\nevance, using the following prompt:, an answer as(q). When building a RAG system,\\nwe usually do not have access to human-annotated\\ndatasets or reference answers. We therefore fo-\\ncus on metrics that are fully self-contained and\\nreference-free. We focus in particular three quality\\naspects, which we argue are of central importance.\\nFirst, Faithfulness refers to the idea that the an-\\nswer should be grounded in the given context. This\\nis important to avoid hallucinations, and to ensure]</td>\n      <td>[Faithfulness refers to the idea that the answer should be grounded in the given context.]</td>\n      <td>Faithfulness refers to the idea that the answer should be grounded in the given context.</td>\n      <td>0.833333</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.953604</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>How many pages are included in the WikiEval dataset, and which years do they cover information from?</td>\n      <td>The WikiEval dataset includes 50 Wikipedia pages covering events that have happened since the start of 2022.</td>\n      <td>[which we refer to as WikiEval4. To construct the\\ndataset, we first selected 50 Wikipedia pages cov-\\nering events that have happened since the start of\\n20225. In selecting these pages, we prioritised\\nthose with recent edits. For each of the 50 pages,\\nwe then asked ChatGPT to suggest a question that\\ncan be answered based on the introductory section\\nof the page, using the following prompt:\\nYour task is to formulate a question from\\ngiven context satisfying the rules given\\nbelow:, which are annotated with human judgments. We\\ncan then verify to what extent our metrics agree\\nwith human assessments of faithfulness, answer\\nrelevance and context relevance. Since we are not\\naware of any publicly available datasets that could\\nbe used for this purpose, we created a new dataset,\\nwhich we refer to as WikiEval4. To construct the\\ndataset, we first selected 50 Wikipedia pages cov-\\nering events that have happened since the start of\\n20225. In selecting these pages, we prioritised, cessing (EMNLP-IJCNLP), pages 563–578, Hong\\nKong, China. Association for Computational Lin-\\nguistics.\\nA\\nExamples from WikiEval\\nTables 2, 3 and 4 show examples from the WikiEval\\ndataset, focusing in particular on answers with high\\nand low faithfulness (Table 2), high and low answer\\nrelevance (Table 3), and high and low context rele-\\nvance (Table 4).\\nQuestion\\nContext\\nAnswer\\nWho directed the film Op-\\npenheimer and who stars\\nas J. Robert Oppenheimer\\nin the film?]</td>\n      <td>[ To construct the dataset, we first selected 50 Wikipedia pages covering events that have happened since the start of 2022.]</td>\n      <td>To construct the dataset, we first selected 50 Wikipedia pages covering events that have happened since the start of 2022.</td>\n      <td>1.000000</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.728021</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Why is evaluating Retrieval Augmented Generation (RAG) systems challenging?</td>\n      <td>Evaluating Retrieval Augmented Generation (RAG) systems is challenging because there are several dimensions to consider. These include the ability of the retrieval system to identify relevant and focused context passages, the ability of the Language Model (LLM) to exploit such passages faithfully, and the quality of the generation itself. Additionally, the overall performance of RAG systems can be affected by various factors such as the retrieval model, the considered corpus, the LM, or the prompt formulation, among others. This complexity makes evaluating RAG systems a challenging task.</td>\n      <td>[retrieval-augmented systems is thus paramount. In\\npractice, RAG systems are often evaluated in terms\\nof the language modelling task itself, i.e. by mea-\\nsuring perplexity on some reference corpus. How-\\never, such evaluations are not always predictive\\nof downstream performance (Wang et al., 2023c).\\nMoreover, this evaluation strategy relies on the LM\\nprobabilities, which are not accessible for some\\nclosed models (e.g. ChatGPT and GPT-4). Ques-\\ntion answering is another common evaluation task,, that are only available through APIs.\\nWhile the usefulness of retrieval-augmented\\nstrategies is clear, their implementation requires\\na significant amount of tuning, as the overall per-\\nformance will be affected by the retrieval model,\\nthe considered corpus, the LM, or the prompt for-\\nmulation, among others. Automated evaluation of\\nretrieval-augmented systems is thus paramount. In\\npractice, RAG systems are often evaluated in terms\\nof the language modelling task itself, i.e. by mea-, Abstract\\nWe introduce RAGAS (Retrieval Augmented\\nGeneration Assessment), a framework for\\nreference-free evaluation of Retrieval Aug-\\nmented Generation (RAG) pipelines.\\nRAG\\nsystems are composed of a retrieval and an\\nLLM based generation module, and provide\\nLLMs with knowledge from a reference textual\\ndatabase, which enables them to act as a natu-\\nral language layer between a user and textual\\ndatabases, reducing the risk of hallucinations.\\nEvaluating RAG architectures is, however, chal-]</td>\n      <td>[Evaluating RAG architectures is, however, challenging because there are several dimensions to consider: the ability of the retrieval system to identify relevant and focused context passages, the ability of the LLM to exploit such passages in a faithful way, or the quality of the generation itself.]</td>\n      <td>Evaluating RAG architectures is, however, challenging because there are several dimensions to consider: the ability of the retrieval system to identify relevant and focused context passages, the ability of the LLM to exploit such passages in a faithful way, or the quality of the generation itself.</td>\n      <td>1.000000</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.941939</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}